import requests
from bs4 import BeautifulSoup

url = f"https://www.zongheng.com/api2/catefine/storeSearch"
for page in range(1, 501):
    print(f"正在爬取第{page}页111111111111111111111")
    data = {
        'worksTypes': '0',
        'bookType': '1',
        'subWorksTypes': '0',
        'totalWord': '0',
        'serialStatus': '9',
        'vip': '9',
        'totalWold': '0',
        'pageNum': f'{page}',
        'pageSize': '20',
        'categoryId': '0',
        'categoryPid': '0',
        'order': 'weekOrder',
        'naodongFilter': '0',
    }
    res = requests.post(url, data=data)
    if res.status_code == 200:
        items = res.json()['result']['bookList']
        for item in items:

            book = {
                "name": item['name'],
                "description": item['description']
            }
            print(f"正在爬取第{book}222222222222222222")

            chapter_res = requests.get(f"https://huayu.zongheng.com/showchapter/{item['bookId']}.html")
            soup = BeautifulSoup(chapter_res.text, features='lxml')
            chapter_list = soup.find("ul", attrs={"class": "chapter-list"})
            for li in chapter_list.find_all("li"):
                if li.get("class") == ["vip", "col-4"]:
                    pass
                else:
                    info = li.find("a")
                    chapter_href = info.get("href")
                    chapter_title = info.get_text()
                    # print(chapter_href, chapter_title)
                    print(f"正在爬取章节{chapter_href}")

                    content_res = requests.get(chapter_href)
                    content_soup = BeautifulSoup(content_res.text, features="lxml")
                    content_container = content_soup.find("div", attrs={"class": "content"})
                    content_ps = content_container.find_all("p")
                    article = "\n".join(content_p.text for content_p in content_ps)
                    print(f"章节爬取成功：{chapter_title}   正文{article}、、、、、、、、、")

        # break
    # 只取第一页
    break
